Smart BB-8 Loves Signing !

Work Distribution

Project group picture

Shiyin Tang

st966@cornell.edu

Designed the linetracker module, audio module, camera deploy module and server module.

Lijuan Hu

lh665@cornell.edu

Designed the obstacle module, android module and server module.

Code Appendix

                #Obstacle module
                # SPDX-FileCopyrightText: 2018 Tony DiCola for Adafruit Industries
                # SPDX-License-Identifier: MIT
                
                # Demo of reading the range and lux from the VL6180x distance sensor and
                # printing it every second.
                
                import time
                
                import board
                import busio
                from motor import Motor,Motor_Line
                from audio import Audio
                
                import adafruit_vl6180x
                
                import time,threading
                
                event=threading.Event()
                
                class Obstacle:
                    def start():
                        event.set()
                        print("event clean, start obstacle")
                
                    def stop():
                        event.clear()
                        Motor.stop()
                        print("this is setting the stop!!!!!") 
                
                    # @staticmethod
                    def obstacle():
                        while True:
                            if not event.is_set():
                                print("[%s] stopping ......")
                                Motor.stop()
                                event.wait()
                                # time.sleep(1)
                            else:
                
                                i2c = busio.I2C(board.SCL, board.SDA)
                                sensor = adafruit_vl6180x.VL6180X(i2c)
                                range_mm = sensor.range
                                print("Range: {0}mm".format(range_mm))
                            
                                print("this is run---")
                                while sensor.range <= 200:
                                        Audio.obstacle()
                                        Motor.back()
                                        time.sleep(2)
                                        Motor.left()
                                        time.sleep(2)
                
                                Motor.forward()     
                                light_lux = sensor.read_lux(adafruit_vl6180x.ALS_GAIN_1)
                                # Delay for a second.
                                time.sleep(1)
                    ob = threading.Thread(target=obstacle,)
                    ob.start()
                    
  
                
              

                #Line tracker module
                import RPi.GPIO as GPIO
                import time
                from motor import Motor_Line,Motor
                import sys
                # import time.threading
                
                TrackPin1 = 21 #line tracker DO
                TrackPin2 = 22 #number 2 line tracker D0
                LedPin   = 19 # LED pin is 19
                
                run = True
                # def setup():
                
                
                def loop():
                    # while True:
                    GPIO.setmode(GPIO.BCM)
                    GPIO.setup(TrackPin1, GPIO.IN, pull_up_down=GPIO.PUD_UP)
                    GPIO.setup(TrackPin2, GPIO.IN, pull_up_down=GPIO.PUD_UP)
                    GPIO.setup(26, GPIO.OUT) # set GPIO 26 as output for left motor 
                    # GPIO.output(26, GPIO.LOW)   
                    GPIO.setup(16, GPIO.OUT) # set GPIO 16 as output for right motor
                    # GPIO.output(16, GPIO.LOW) 
                    if GPIO.input(TrackPin1) == GPIO.LOW and GPIO.input(TrackPin2) == GPIO.LOW:
                        # print('Both White')  #此时那个绿色的等亮
                        Motor_Line.forward()
                        # GPIO.output(LedPin, GPIO.LOW)  # white line, led off #等弄完了再加，记得加入！！！！
                    elif(GPIO.input(TrackPin1) == GPIO.LOW and GPIO.input(TrackPin2) == GPIO.HIGH):
                        # print('2 Black')
                        Motor_Line.left()
                    elif(GPIO.input(TrackPin1) == GPIO.HIGH and GPIO.input(TrackPin2) == GPIO.LOW):
                        Motor_Line.right()
                        # GPIO.output(LedPin, GPIO.HIGH) # led on
                    elif(GPIO.input(TrackPin1) == GPIO.HIGH and GPIO.input(TrackPin2) == GPIO.HIGH):
                        Motor_Line.left()
                        
                def destroy():
                    # GPIO.output(LedPin, GPIO.LOW)     # led off
                    GPIO.cleanup()                     # Release resource
                
                GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
                
                
                GPIO.setup(TrackPin1, GPIO.IN, pull_up_down=GPIO.PUD_UP)
                GPIO.setup(TrackPin2, GPIO.IN, pull_up_down=GPIO.PUD_UP)
                
                class Tracker:
                    @staticmethod
                    def start():
                     
                        try:
                            # print("start the line tracking------->")
                            loop()
                        except KeyboardInterrupt: 
                        # When 'Ctrl+C' is pressed, the child program destroy() will be  executed.
                            destroy()
                    @staticmethod
                    def stop():
                        # sys.exit()
                        destroy()
                       

              

                #motor
                import time
import RPi.GPIO as GPIO
import threading
import os
GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
GPIO.setup(26, GPIO.OUT) # set GPIO 26 as output for left motor 
GPIO.output(26, GPIO.LOW)   
GPIO.setup(16, GPIO.OUT) # set GPIO 16 as output for right motor
GPIO.output(16, GPIO.LOW)    

#clk & counter-clk:
fc = 1/0.0210 #3
dc = 100/21.0
fcc = 1/0.0220 #217
dcc = 200/22.0
#slower
fc_l = 1/0.02144  #214
dc_l = 144/21.44
fcc_l = 1/0.02154 #216
dcc_l = 154/21.54

# To create a PWM instance:
pa = GPIO.PWM(26, fc)
pa.start(0)
pb = GPIO.PWM(16, fc)
pb.start(0)

# time.sleep(10)
# GPIO.cleanup()

class Motor:
	@staticmethod
	def forward():
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		pa.ChangeDutyCycle(dcc)
		pa.ChangeFrequency(fcc)
		pb.ChangeDutyCycle(dc)
		pb.ChangeFrequency(fc)
		Info.p('Car Forwarding--->')
	@staticmethod
	def back():
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		# start = time.time()
		# while ( time.time() - start < 0.5 ) :
		pa.ChangeDutyCycle(dc)
		pa.ChangeFrequency(fc)
		pb.ChangeDutyCycle(dcc)
		pb.ChangeFrequency(fcc)
		Info.p('Car Backing--->')
	@staticmethod
	def left():
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		# start = time.time()
		# while ( time.time() - start < 0.5 ) :
		pa.ChangeDutyCycle(0)
		pa.ChangeFrequency(1/0.0215)
		pb.ChangeDutyCycle(dc)
		pb.ChangeFrequency(fc)
		Info.p('Car Turning Left--->')
	@staticmethod
	def right():
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		# start = time.time()
		# while ( time.time() - start < 0.5 ) :
		pa.ChangeDutyCycle(dcc)
		pa.ChangeFrequency(fcc)
		pb.ChangeDutyCycle(0)
		pb.ChangeFrequency(1/0.0215)
		Info.p('Car Turning Right--->')
	@staticmethod
	def stop():
		# start = time.time()
		# while ( time.time() - start < 0.5 ) :
		pa.ChangeDutyCycle(0)
		pa.ChangeFrequency(1/0.0215)
		pb.ChangeDutyCycle(0)
		pb.ChangeFrequency(1/0.0215)
		Info.p('Car Stopping--->')
		

class Info:
	@staticmethod
	def p(message):
		print ('Info: '+message) 


class Motor_Line:
	@staticmethod
	def forward():
		Info.p('Line forward--->')
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		pa.ChangeDutyCycle(dcc_l)
		pa.ChangeFrequency(fcc_l)
		pb.ChangeDutyCycle(dc_l)
		pb.ChangeFrequency(fc_l)
		# time.sleep(5)
	@staticmethod
	def right():
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		pa.ChangeDutyCycle(dcc_l)
		pa.ChangeFrequency(fcc_l)
		pb.ChangeDutyCycle(0)
		pb.ChangeFrequency(1/0.0215)
		Info.p('Line right----->')
	@staticmethod
	def left():
		GPIO.setmode(GPIO.BCM) # Set for broadcom numbering not board numbers...
		GPIO.setup(26, GPIO.OUT)   
		GPIO.setup(16, GPIO.OUT) 
		pa.ChangeDutyCycle(0)
		pa.ChangeFrequency(1/0.0215)
		pb.ChangeDutyCycle(dc_l)
		pb.ChangeFrequency(fc_l)
		Info.p('Line left------>')



              

                #server
                from socket import *
import sys
import time
from motor import Motor
from audio import Audio
from line_tracker import Tracker
import RPi.GPIO as GPIO
from obstacle import Obstacle
# import rpi_camera_surveillance_system

commands ={
  'H':Motor.stop,
  'DirForward':Motor.forward,
  'DirBack':Motor.back, 
  'DirStop':Motor.stop,
  'DirLeft':Motor.left,
  'DirRight':Motor.right,
  'L':Tracker.start,
  'quit':Obstacle.stop,
  'O': Obstacle.start
}

sounds={
  'H':Audio.BB8,
  'DirForward':Audio.updown,
  'DirBack':Audio.updown, 
  'DirStop':Audio.stop,
  'DirLeft':Audio.leftright,
  'DirRight':Audio.leftright,
  'L':Tracker.start,
  'quit':Obstacle.stop,
  'O':Obstacle.start
}

# Audio.updown()
def playsound(command):
  print(command)
  sounds[command]()

def execute(command):	
	commands[command]()

HOST ='10.49.241.19' #the ip of rapberry pi
PORT = 8701
s= socket(AF_INET, SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
print ('listening on 8701')

import time,threading

event=threading.Event()
operation = "DirForward"
moduleEvent = threading.Event()
mod = "L"

def sou():
   while True:
        if event.is_set():
            # print("[%s] playing sound ......" )
            global operation
            sounds[operation]()
            if operation == "quit":
              # os.system("pkill mplayer")
              Obstacle.stop()
              Tracker.stop()
              
        else:
            # print("[%s]  waiting for the audio setting..")
            event.wait()
            Audio.quit()

def car():
    while 1:
      conn, addr = s.accept()
      print ('Connected by:', addr)
      while 1:
              command= conn.recv(1024)
              command = str(command, 'UTF-8')
              if not command or command not in commands: 
                  event.clear()
                  # print('not command--->')
                  break

              if command == "quit":
                print("this is the stop command **************")

              if command == "DirStop":
                # print("this is the stop command **************")
                event.clear()
                
              else: 
                # print("not stopping command")
                event.set()  #play sound event
                global operation
                operation = command  
              execute(command)
              
      conn.close()

sou = threading.Thread(target=sou,)
sou.start()


car1=threading.Thread(target=car,)
car1.start()

              

                #audio

import subprocess
import os
import sys
import time

# from omxplayer.player import OMXPlayer
# from pathlib import Path
# from time import sleep
 
# up_path = Path("obstacle_aoo.mp3") 

# player_up = OMXPlayer(up_path)       
# sleep(1)                   
# player_up.quit()


paths ={
  'obstacle':"obstacle_aoo.mp3",
  'updown':"updown.mp3", 
  'leftright':"leftright.mp3",
  'stop':"stop.mp3",
  'H':"BB8video.mp4"
}

# path_obstacle = "obstacle_aoo.mp3"

# Use a python os call to execute some 'echo' commands...
class Audio:
    def quit():
        print('quit-->')
        # player_up.quit()
        os.system("mplayer q")

    @staticmethod   
    def BB8():
        # thread = threading.Thread(target = BB8)
        # player_up = OMXPlayer(up_path)  
        os.system('sudo SDL_VIDEODRIVER=fbcon SDL_FBDEV=/dev/fb0 mplayer -vo sdl -framedrop -zoom -x 320 -y 220 %s' % paths['H'])
        print('here play BB8') #对应updown.mp3
        # time.sleep(1)
    @staticmethod
    def updown():
        # player_up = OMXPlayer(up_path)  
        os.system('mplayer %s' % paths['updown'])
        print('here play forward audio') #对应updown.mp3
        # time.sleep(1)
   
    @staticmethod
    def leftright():
        os.system('mplayer %s' % paths['leftright'])
        print('here play left audio') #对应leftright.mp3
   
    @staticmethod
    def stop():
        os.system('mplayer %s' % paths['stop'])
        print('here play stop audio') #对应stop.mp3

    @staticmethod
    def obstacle():
        os.system('mplayer %s' % paths['obstacle'])
        # os.system ("echo 'seek +10' > /home/pi/video_fifo")
        print('here play obstacle audio') #对应obstacle_aoo.mp3
    
    @staticmethod
    def mute():
        print("no play")

    @staticmethod
    def mute1():
        print("mute1")

# v=input()
 # if (v == "play"):
class Play:
    @staticmethod
    def play():
        Audio.updown()
        time.sleep(1)
        Audio.quit()
              

                #Android main page
                package com.example.myapplication1;


import android.app.Activity;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.content.ServiceConnection;
import android.os.*;
import android.view.MotionEvent;
import android.view.View;
import android.webkit.WebChromeClient;
import android.webkit.WebSettings;
import android.widget.Button;
import android.widget.Toast;
import androidx.appcompat.app.AppCompatActivity;
import android.webkit.WebView;
import android.webkit.WebViewClient;


public class MainActivity extends AppCompatActivity{
    private String host;
    private int port;
    public static Context context;
    private WebView webview;
    private boolean booleanConnect = false;
    boolean longPress = false;
    public TcpCommandService.SendCommandBinder sendCommandBinder=null; //Use this binder to notify service sending command data
    private ServiceConnection tcpConnection=new ServiceConnection() { //The connection between Main activity and TcpCommandService
        @Override
        public void onServiceConnected(ComponentName name, IBinder service) {
            sendCommandBinder=(TcpCommandService.SendCommandBinder)service;
        }

        @Override
        public void onServiceDisconnected(ComponentName name) {
        }
    };

    public UDPFrameRecService.NoticeBinder noticeUDPBinder=null;//Use this binder to notify UDP service
    private ServiceConnection udpConnection=new ServiceConnection() {
        @Override
        public void onServiceConnected(ComponentName name, IBinder service) {
            noticeUDPBinder=(UDPFrameRecService.NoticeBinder)service;
        }

        @Override
        public void onServiceDisconnected(ComponentName name) {

        }
    };

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        longPress = false;
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        //context=getApplicationContext();
        context=this;//Get the context of this activity so that the static functions can use
        webview =(WebView)findViewById(R.id.webView);

        webview.setWebViewClient(new WebViewClient());
        webview.getSettings().setJavaScriptEnabled(true);
        webview.getSettings().setDomStorageEnabled(true);
        webview.setOverScrollMode(WebView.OVER_SCROLL_NEVER);
        WebSettings settings = webview.getSettings();

        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) {
            settings.setMixedContentMode(WebSettings.MIXED_CONTENT_ALWAYS_ALLOW);
        }
        settings.setLoadsImagesAutomatically(true);

        settings.setUseWideViewPort(true);
        settings.setLoadWithOverviewMode(true);

        webview.setWebChromeClient(new WebChromeClient() );
        webview.loadUrl("http://10.49.241.19:8000/index.html");

        Intent startInfoIntent=getIntent();
        host=startInfoIntent.getStringExtra("tcpHost");
        port=startInfoIntent.getIntExtra("tcpPort",50);

        Intent startTcpServiceIntent=new Intent(this,TcpCommandService.class);
        startTcpServiceIntent.putExtra("tcpHost",host);
        startTcpServiceIntent.putExtra("tcpPort",port);

        //Start and bind TcpCommandService
        startService(startTcpServiceIntent);
        bindService(startTcpServiceIntent,tcpConnection,BIND_AUTO_CREATE);
        initButtons();


    }

    @Override
    protected void onDestroy() {
        super.onDestroy();
        unbindService(tcpConnection);
        stopService(new Intent(this,TcpCommandService.class));

    }


    /**
     * Init the Direction Buttons and Camera Buttons, set OnTouchListener for these buttons so 
     * command can be sent when buttons are down or up.
     */
    public void initButtons()
    {
        Button buttonQuit=(Button)findViewById(R.id.buttonQuit);
        buttonQuit.setOnTouchListener(new ButtonListener("quit","quit"));

        Button buttonLine=(Button)findViewById(R.id.buttonLineTracker);
        buttonLine.setOnTouchListener(new ButtonListener("L","L"));

        Button buttonObs=(Button)findViewById(R.id.buttonObstacle);
        buttonObs.setOnTouchListener(new ButtonListener("O","O"));

        Button buttonHis=(Button)findViewById(R.id.buttonHistory);
        buttonHis.setOnTouchListener(new ButtonListener("H","H"));

        Button buttonDirFor=(Button)findViewById(R.id.buttonDirForward);
        buttonDirFor.setOnTouchListener(new ButtonListener("DirStop","DirForward"));
        Button buttonDirBack=(Button)findViewById(R.id.buttonDirBack);
        buttonDirBack.setOnTouchListener(new ButtonListener("DirStop","DirBack"));
        Button buttonDirLeft=(Button)findViewById(R.id.buttonDirLeft);
        buttonDirLeft.setOnTouchListener(new ButtonListener("DirStop","DirLeft"));
        Button buttonDirRight=(Button)findViewById(R.id.buttonDirRight);
        buttonDirRight.setOnTouchListener(new ButtonListener("DirStop","DirRight"));
        Button buttonDirStop=(Button)findViewById(R.id.buttonDirStop);
        buttonDirStop.setOnTouchListener(new ButtonListener("DirStop","DirStop"));
        Button buttonClose=(Button)findViewById(R.id.buttonClose);
        buttonClose.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                sendCommandBinder.closeOrShutdown("Close"); //Notify the car this client will close the connection
            }
        });

        Button buttonShutdown=(Button)findViewById(R.id.buttonShutdown);
        buttonShutdown.setOnClickListener(new View.OnClickListener() {
            @Override
            public void onClick(View v) {
                sendCommandBinder.closeOrShutdown("Shutdown"); //Notify the car to shutdown itself
            }
        });
    }

    /**
     * When a control button is down or up, send the command 
     */
    class ButtonListener implements View.OnTouchListener{
        private String buttonUpCommand; //Specify the command when button is up
        private String buttonDownCommand; //Specify the command when button is down
        public ButtonListener(String buttonUpCommand,String buttonDownCommand){
            this.buttonUpCommand=buttonUpCommand;
            this.buttonDownCommand=buttonDownCommand;
        }
        @Override
        public boolean onTouch(View v, MotionEvent event) {
            longPassCmd(event);
            return true;
        }
        private void longPassCmd( MotionEvent event){
            switch (event.getAction()){

                case MotionEvent.ACTION_DOWN:{
                    longPress = true;
                    Thread longPressThread = new Thread(){
                        public void run(){
                            super.run();
                            while(true){
                                if(longPress == true){
                                    try{
                                        sendCommandBinder.sendCommand(buttonDownCommand);
                                        Thread.sleep(1000);
                                    }catch (InterruptedException e){
                                        e.printStackTrace();
                                    }
                                }else{
                                    break;
                                }
                            }
                        }
                    };
                    longPressThread.start();
                    break;
                }
                case MotionEvent.ACTION_UP:
                    longPress = false;
                    sendCommandBinder.sendCommand(buttonUpCommand);
                    break;

                default:
                    break;
            }
        }
    }


    public static final int TOAST_TEXT=1;
    public static final int CLOSE_OR_SHUTDOWN=2;
    public static final int UPDATE_IMAGEVIEW=3;
    /**
     * Use the handler to update main activity according to service's message
     */
    public static Handler updateUIHandler=new Handler(){
        @Override
        public void handleMessage(Message msg) {
            switch (msg.what){
                case TOAST_TEXT: //Display the toast according to the message from service
                    Toast.makeText(context,(String)msg.obj,Toast.LENGTH_SHORT).show();
                    break;
                case CLOSE_OR_SHUTDOWN: //The tcp service has sent close or shutdown command, and the main activity should exit now
                    ((MainActivity)context).finish();
                    break;
              
                default:
                    break;
            }
        }
    };
}